/**
 * Project  : Looker
 * Package  : net.freestu.looker.util
 * File     : Importer.java
 * Author   : solosky < solosky772@qq.com >
 * Date     : 2009-9-14
 * Modified : 2009-9-14
 * License  : Apache License 2.0 
 */
package net.freestu.looker.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;

import net.freestu.looker.common.Category;
import net.freestu.looker.common.Looker;
import net.freestu.looker.common.NewsEntity;
import net.freestu.looker.common.NewsMeta;
import net.freestu.looker.common.Site;
import net.freestu.looker.common.SiteManager;
import net.freestu.looker.monitor.FetcherException;
import net.freestu.looker.monitor.HtmlFetcher;
import net.freestu.looker.monitor.ListParser;
import net.freestu.looker.monitor.ModifyDispatcher;
import net.freestu.looker.monitor.NewsArchiver;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.log4j.Logger;

/**
 *  
 */
public class CdutIndexImporter
{

	/**
	 * 启动对象
	 */
	private Starter starter;
	
	/**
	 * 提取的新闻信号量
	 * 用于限制最多同时进行提取新闻的线程
	 */
	private Semaphore activeArchiverSemaphore;
	
	/**
	 * 需导入的新闻栏目对象
	 */
	private ArrayList<Category> importCategoryList;
	
	/**
	 * 完成的抓取器计数器
	 */
	private Integer atomicInt;
	
	/**
	 *  日志记录
	 */
	private static Logger logger = Logger.getLogger(CdutIndexImporter.class);
	
	
	/**
	 * 初始化
	 * @param lookerXmlConfig
	 * @throws ConfigurationException
	 */
	
	public void init(String lookerXmlConfig, int maxArchiver) throws ConfigurationException
	{
		starter = new Starter();
		starter.loadConfiguration(lookerXmlConfig);
		starter.registerListener();
		activeArchiverSemaphore = new Semaphore(maxArchiver);
		importCategoryList = new ArrayList<Category>();
		atomicInt = new Integer(0);
		
	}
	
	/**
	 * 找出需要导入的栏目
	 */
	public void findImportCategory()
	{
		Iterator<Site> sit = SiteManager.getInstance().getSiteList().iterator();
		while(sit.hasNext()) {
			Site site = sit.next();
			Iterator<Category> cit = site.getCategoryList().iterator();
			while(cit.hasNext()) {
				Category cate = cit.next();
				if(cate.getConfiguration().getString("importPage")!=null) {
					importCategoryList.add(cate);
				}
			}
		}
	}
	
	/**
	 * 确认导入栏目任务
	 * @return
	 */
	public boolean confirmImportCategory()
	{
		System.out.println("请确认导入任务:");
		System.out.println("--------------------------------------------");
		System.out.println("站点\t栏目\t导入页数\t首页地址");
		System.out.println("--------------------------------------------");
		Iterator<Category> cit = importCategoryList.iterator();
		while(cit.hasNext()) {
			Category cate =cit.next();
			System.out.println(cate.getSite().getTitle()+'\t'
					+cate.getTitle()+'\t'
					+cate.getConfiguration().getString("importPage")+'\t'
					+cate.getPaging().getURL(1));
		}
		System.out.println("--------------------------------------------");
		System.out.print("确认?(y,n):");
		BufferedReader  reader = new BufferedReader(new InputStreamReader(System.in));
        try {
        	String confirmStr = reader.readLine();
	        if(confirmStr.charAt(0)=='y')
				return true;
	        else
	        	return false;
        } catch (IOException e) {
	       return false;	//never happen..
        }
	}
	
    /**
	 * 等待所有的任务完成
	 * @throws InterruptedException 
	 */
	public void waitAllDone() throws InterruptedException
	{
			//如果完成的抓抓取器数目小于完成数目，就继续等待
		synchronized (atomicInt) {
			while(atomicInt<this.importCategoryList.size())
				atomicInt.wait();
        }
			logger.info("所有的列表抓取线程完成。");
	}
	
	/**
	 * 完成导入
	 */
	public void finishImport()
	{
		Looker.executors.shutdown();
	}
	
	/**
	 * 开始导入
	 */
	public void startImport()
	{
		Iterator <Category> it = this.importCategoryList.iterator();
		while(it.hasNext()) {
			Looker.executors.submit(new NewsMetaArchiveWork(it.next()));
		}
	}
	
	/**
	 * 内部类，提取新闻链接，并交给
	 */
	public class NewsMetaArchiveWork implements Runnable
	{
		/**
		 * 导入的栏目
		 */
		private Category category;
		
		public NewsMetaArchiveWork(Category category)
		{
			this.category = category;
		}
		
		/**
		 * 提取一页的新闻链接
		 * @throws FetcherException 
		 */
		@SuppressWarnings("unchecked")
        private ArrayList<NewsMeta> archiveList(int page) throws FetcherException
		{
			HtmlFetcher fetcher = new HtmlFetcher(category.getPaging().getURL(page));
			fetcher.fetch();
			String html = (String) fetcher.getFeched();
			ListParser parser = new ListParser(html, category);
			parser.parse();
			return (ArrayList<NewsMeta>) parser.getParsed();
		}
		
		/**
		 * 分发新闻链接
		 * @throws InterruptedException 
		 */
		private void dispatchNewsMeta(NewsMeta newsMeta) throws InterruptedException
		{
			activeArchiverSemaphore.acquire();
			Looker.executors.submit(new NewsArchiveWork(newsMeta));
		}


        public void run()
        {
        	int importPage = category.getConfiguration().getInt("importPage");
        	for(int i=0; i<importPage; i++) {
        		try {
	                ArrayList<NewsMeta> newsMetaList = archiveList(i);
	                Iterator <NewsMeta> it = newsMetaList.iterator();
	                while(it.hasNext()) {
	                	dispatchNewsMeta(it.next());
	                }
	                
                } catch (FetcherException e) {
	               logger.warn("提取新闻列表时出错:"+e.getMessage());
                } catch (InterruptedException e) {
                	logger.warn("提交新闻正文提取任务事出错:"+e.getMessage());
                }
                synchronized (atomicInt) {
	               atomicInt++;
	               atomicInt.notifyAll();
                }
        		logger.info("新闻列表下载完成:"+category);
        		
        	}
	        
        }
    
        
        /**
         * 内部类，下载新闻正文和附件
         */
        public class NewsArchiveWork implements Runnable
        {
        	private NewsMeta newsMeta;
            
        	public NewsArchiveWork(NewsMeta newsMeta)
        	{
        		this.newsMeta = newsMeta;
        	}
        	
        	public void run()
            {
        		logger.info("开始下载新闻："+newsMeta);
	            Future<NewsEntity> future = Looker.executors.submit(new NewsArchiver(newsMeta));
	            NewsEntity newsEntity = null;
	            try {
	               newsEntity = future.get();
                } catch (Exception e) {
                	logger.error("NewsArchiver错误:"+e);
                }
                if(newsEntity!=null) {
                	ModifyDispatcher.getInstance().newsAdded(newsEntity);
                }
                activeArchiverSemaphore.release();
                logger.info("成功下载新闻："+newsMeta);
            }
        	
        }
		
	}
	
	/**
	 * @param args
	 * @throws ConfigurationException 
	 * @throws NumberFormatException 
	 * @throws InterruptedException 
	 */
	public static void main(String[] args) throws NumberFormatException, ConfigurationException, InterruptedException
	{
		if(args.length<2) {
			System.out.println("Usage: lookerConfigFile maxActiveNewsArchiver");
			System.out.println("Ex: ./conf/looker.xml 10");
			return;
		}
		
		CdutIndexImporter importer = new CdutIndexImporter();
		importer.init(args[0], Integer.parseInt(args[1]));
		importer.findImportCategory();
		if(importer.confirmImportCategory()) {
			importer.startImport();
			importer.waitAllDone();
			importer.finishImport();
		}

	}

}
